package com.g2room.driving.grap.article.kc;

import java.io.IOException;
import java.net.MalformedURLException;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;

import com.g2room.driving.grap.GenericGrap;
import com.g2room.driving.model.po.ArticleGrap;
import com.g2room.driving.util.DbUtil;
import com.g2room.driving.util.IpUtil;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.DomElement;
import com.gargoylesoftware.htmlunit.html.HtmlAnchor;
import com.gargoylesoftware.htmlunit.html.HtmlListItem;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlSpan;


public class GrabQQlawDataByProxy {
	static Connection connection = null;
	static PreparedStatement  ps = null;
	static Map<String, String> errorUrls = new HashMap<String, String>();
	static List<ArticleGrap> articleList = new ArrayList<ArticleGrap> ();
	private static final String homePage = "http://auto.qq.com/l/service/policy/li.htm";
	private static final String soruce = "腾讯汽车";
	
	private static Connection getConn() throws ClassNotFoundException, SQLException {
		if(null == connection) {
			connection = DbUtil.getConn();
		}
		return connection;
	}
	static Map<String, String> errMap = new HashMap<String, String>();
	public static void main(String[] args) throws Exception{
		GrabQQlawDataByProxy gkd = new GrabQQlawDataByProxy();
		gkd.getNav(homePage);
		// 最后输出失败的链接到数据库
		insertError2Db();
    }
	
	
	/**
	 * 获取头部导航菜单
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getNav(String url) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
        WebClient wc = GenericGrap.getWebClient();
		//获取页面
        HtmlPage page = null;
        try {
        	page = wc.getPage(url);
        } catch (Exception e) {
        	errorUrls.put(url, "导航页");
        	e.printStackTrace();
        	return;
        }
        //查找并获取特定input
        List<?> list = page.getByXPath("//div[@id=\"autonav_yongche\"]/ul/li/a");
        Map<String, String> pages = new HashMap<String, String>();
        for(Object hs : list) {
        	HtmlAnchor htmlAnchor = (HtmlAnchor) hs;
        	pages.put(htmlAnchor.getAttribute("href"), htmlAnchor.asText());
        }
        if (null != wc) wc.close();
        
        for(String key : pages.keySet()) {
        	if(key.equals("全部")) continue;
	        try {
	        	Thread.sleep(IpUtil.genRandomNum() * 500);
	        } catch (InterruptedException e) {
	        	e.printStackTrace();
	        }
	        getPage(homePage + key, pages.get(key));
        }
	}
	
	/**
	 * 获取分页数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getPage(String url, String typeName) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get page data!" + typeName);
		WebClient wc = GenericGrap.getWebClient();
		//获取页面
		HtmlPage htmlPage = null;
        try {
        	htmlPage = wc.getPage(url);
        } catch (Exception e) {
        	errorUrls.put(url, typeName);
        	e.printStackTrace();
        	return;
        }
		// 获取分页数据
		List<?> list = htmlPage.getByXPath("//div[@class=\"pageNav\"]/a");
		List<String> hrefs = new ArrayList<String>();
		int page = 1;
		HtmlAnchor htmlAnchor;
		hrefs.add(url);
        if(list.size() > 1) {
        	htmlAnchor = (HtmlAnchor) list.get(list.size() - 2);
        	page = Integer.valueOf(htmlAnchor.asText());
        	String pageHref = htmlAnchor.getAttribute("href").replace(page + ".html", "###.html");
        	for( int i = 2; i < page+1; i++) {
        		hrefs.add(pageHref.replace("###", i+""));
        	}
        }
        
        if (null != wc) wc.close();
        
        for(String href : hrefs) {
        	try {
				Thread.sleep(IpUtil.genRandomNum() * 500);
			} catch (InterruptedException e) {
				e.printStackTrace();
			}
			getList(homePage + href, typeName);
        }
	}
	
	/**
	 * 获取列表数据
	 * @param url
	 * @throws FailingHttpStatusCodeException
	 * @throws MalformedURLException
	 * @throws IOException
	 */
	public void getList(String url, String typeName) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get list data!url==  " + url + "   ,typeName=" + typeName);
		WebClient wc = GenericGrap.getWebClient();
		//获取页面
		HtmlPage page = null;
        try {
        	page = wc.getPage(url);
        } catch (Exception e) {
        	errorUrls.put(url, typeName);
        	e.printStackTrace();
        	return;
        }
		// 获取分页数据
		List<?> list = page.getByXPath("//div[@class=\"mod newslist\"]/ul/li");
		Map<String, ArticleGrap> details = new HashMap<String, ArticleGrap>();
		for(Object hs : list) {
			HtmlListItem li = (HtmlListItem) hs;
			List<?> hrefs = li.getByXPath("a[@target=\"_blank\"]");
			List<?> spans = li.getByXPath("span");
			for (Object a : hrefs) {
				HtmlAnchor link =  (HtmlAnchor) a;
				ArticleGrap article = new ArticleGrap();
//				articles.setTypeId(typeId);
				article.setTypeName(typeName);
				String href = link.getHrefAttribute();
				article.setSourceUrl(href);
				article.setTitle(link.asText());
				article.setPublishTime(((HtmlSpan) spans.get(0)).asText());
				details.put(href, article);
				System.out.println(article.getTitle() + "==" + typeName + "===" + href);
			}
		}
		System.out.println(typeName + "==文章数=" + details.size());
        if (null != wc) wc.close();
//        
//        for(String key : details.keySet()) {
//        	try {
//        		Thread.sleep(IpUtil.genRandomNum() * 500);
//        	} catch (InterruptedException e) {
//        		e.printStackTrace();
//        	}
//        	getDetail(key, details.get(key));
//        }
	}
	
	public void getDetail(String url, ArticleGrap articles) throws FailingHttpStatusCodeException, MalformedURLException, IOException {
		System.out.println("start get detail data!  " + url + ",  title=" + articles.getTitle());
		WebClient wc = GenericGrap.getWebClient();
		HtmlPage page = null;
        try {
        	page = wc.getPage(url);
        } catch (Exception e) {
        	articleList.add(articles);
        	e.printStackTrace();
        	return;
        }

        articles.setSourceUrl(url);
		// 获取分页数据
		DomElement div = page.getElementById("news_content");
//		List<DomNode> nodes = div.getChildNodes();
//		System.out.println(div.asXml());
//		div.removeChild(nodes.get(nodes.size()));
//		div.removeChild(nodes.get(nodes.size() - 1));
//		div.removeChild(nodes.get(nodes.size() - 2));
//		div.removeChild(nodes.get(nodes.size() - 3));
		if(null != div) {
			articles.setContent(div.asXml());
			insert2Db(articles);
		}
        if (null != wc) wc.close();
	}
	int sqlCount = 0;
	public void insert2Db(ArticleGrap articles) {
		try {
			if(sqlCount == 0) ps = getConn().prepareStatement("insert into d_article_grap(title,content,publish_time,type_name,source,source_url, state) values(?,?,?,?,?,?)");
			// title,content,publish_time,type_name,source,source_url
			ps.setString(1, articles.getTitle());
			ps.setString(2, articles.getContent());
			ps.setString(3, articles.getPublishTime());
			ps.setString(4, articles.getTypeName());
			ps.setString(5, soruce);
			ps.setString(6, articles.getSourceUrl());
			ps.execute();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
	
	public static void insertError2Db() {
		try {
			PreparedStatement  ps = getConn().prepareStatement("insert into d_article_grap_err(title,source,source_url,type_name) values(?,?,?,?)");
			for(String url : errorUrls.keySet()) {
				ps.setString(1, errorUrls.get(url));
				ps.setString(2, soruce);
				ps.setString(3, url);
				ps.setString(4, "导航和列表错误");
				ps.addBatch();
			}
			for(ArticleGrap article : articleList) {
				ps.setString(1, article.getTitle());
				ps.setString(2, soruce);
				ps.setString(3, article.getSourceUrl());
				ps.setString(4, article.getTypeName());
				ps.addBatch();
			}
			ps.executeBatch();
			if(null != ps) ps.close();
			if(null != connection) connection.close();
		} catch(Exception e) {
			e.printStackTrace();
		}
	}
}
